# usage: select-by-keywords.py keywords_dir input_file output_file [threshold_general threshold_special]

import os, sys

def ExtendKeyword(line):
    i = 0
    varWord = -1
    variants = []
    words = line.split(" ")
    for word in words:
        if ("|" in word):
            varWord = i
            i += 1
            
    if (varWord != -1):
        for var in words[varWord].split("|"):
            words[varWord] = var
            variants.append(" ".join(words))
    else:
        variants.append(" ".join(words))
            
    return variants


def ReadKeywords(file):
    localKeywords = []
    localTemplates = []
    
    maxPriorityEdge = 0
    maxPriorityEdge_templates = 0
    
    with open(file) as keywordsFile:
        for line in keywordsFile:
            line = line.strip("\n")
            if(line == ""):
                if (maxPriorityEdge != 0):
                    continue
                maxPriorityEdge = len(localKeywords)
                maxPriorityEdge_templates = len(localTemplates)
            if (line[-1:] == "*"):
                localTemplates.append(line[:-1])
            localKeywords.extend(ExtendKeyword(line))
            
    return (localKeywords, localTemplates, maxPriorityEdge, maxPriorityEdge_templates)

def ReadNewsItem(newsFile):
    newsItem = ""
    nextNewsItemHeader = ""
    for line in newsFile:
        line = line.strip("\n")
        if (line == ""):
            continue
        if (line.startswith("/news/")):
            nextNewsItemHeader = line
            break
        
        newsItem += line + "\n"
        
    return (newsItem, nextNewsItemHeader)

def SplitToWords(newsItem):
    return newsItem.replace("\n", " ").replace(".", " ").replace(",", " ").split(" ")

def Classify(newsItem, keywords, templates, priorityEdge, priorityEdge_template):
    categoryCounter = {}
    for category in keywords:
        categoryCounter[category] = 0
    
    wordList = SplitToWords(newsItem)
    for word in wordList:
        if (word == ""):
            continue
        for category in keywords:
            iPriorityTmpl = priorityEdge_template[category]
            if (word in keywords[category]):
                categoryCounter[category] += 1
                print(word)
                if (keywords[category].index(word) < priorityEdge[category]):
                    categoryCounter[category] += 4
                
            for keyword in templates[category]:
                if (word.startswith(keyword)):
                    categoryCounter[category] += 1
                    print (word)
                    if (iPriorityTmpl >= 0):
                        categoryCounter[category] += 4
                iPriorityTmpl -= 1
    
    
    curCategory = ""
    for category, catValue in categoryCounter.items():
        
        if category == "general":
            continue
        
        if (curCategory == "") or (catValue > categoryCounter[curCategory]):
            curCategory = category
        
        #if (curCategory == "" or (categoryCounter[category] > categoryCounter[curCategory] and category != "general")):
        #if (catValue > categoryCounter[curCategory]):
            #curCategory = category
            
    if (categoryCounter[curCategory] > threshold and (categoryCounter[curCategory] + categoryCounter["general"]) > thresholdGeneral):
        return "%s: %d" % (curCategory, categoryCounter[curCategory])
    
    return None
            


if (len(sys.argv) < 4):
    print ("использование: select-by-keywords.py keywords_dir input_file output_file [threshold_general treshold_special]")
    sys.exit(1)
    
if (len(sys.argv) > 4):
    thresholdGeneral = int(sys.argv[4])
    threshold = int(sys.argv[5])
else:
    thresholdGeneral = 10
    threshold = 4

keywords = {}
templates = {}
priorityEdge = {}
priorityEdge_template = {}

if (sys.argv[1][-1:] != "/"):
    sys.argv[1] += "/"
    
for dirname, dirnames, filenames in os.walk(sys.argv[1]):
    for file in filenames:
        if (file == "keywords.txt"):
            (keywords["general"], templates["general"], priorityEdge["general"], priorityEdge_template["general"]) = ReadKeywords(sys.argv[1] + file)
        else:
            (keywords[file.replace("keywords_", "")], templates[file.replace("keywords_", "")], priorityEdge[file.replace("keywords_", "")], priorityEdge_template[file.replace("keywords_", "")]) = ReadKeywords(sys.argv[1] + file)

nextHeader = ""
with open(sys.argv[2]) as newsFile:
    with open(sys.argv[3], "w", encoding = "utf8") as outputFile:
        while (True):
            currentHeader = nextHeader
            (newsItem, nextHeader) = ReadNewsItem(newsFile)
            if (nextHeader == ""):
                break
            if (newsItem == ""):
                continue
            category = Classify(currentHeader + "\n" + newsItem, keywords, templates, priorityEdge, priorityEdge_template)
            if (category != None):
                outputFile.write("-- %s --\n%s\n%s\n\n\n" % (category, currentHeader, newsItem))